package com.shujia.spark

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

object Demo8GroupByKey {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf()
      .setAppName("make")
      .setMaster("local")


    val sc = new SparkContext(conf)


    val studentsRDD: RDD[String] = sc.textFile("spark/data/students.txt")


    val kvRDD: RDD[(String, Int)] = studentsRDD.map(student => {
      val clazz: String = student.split(",")(4)
      (clazz, 1)
    })

    /**
      * groupByKey算子,通过key进行分组
      *
      * 返回值: key是分组的key  value是这个key所有value构成的迭代器(迭代器只能迭代一次)
      *
      */

    val groupRDD: RDD[(String, Iterable[Int])] = kvRDD.groupByKey()


    val numRDD: RDD[(String, Int)] = groupRDD.map {
      case (clazz: String, iter: Iterable[Int]) => {

        //班级人数
        val num: Int = iter.sum

        (clazz, num)
      }
    }


    numRDD.foreach(println)

  }
}
